{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2285f3ec-f1b7-4c53-b8c5-94db8fd0ec74",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "600beaf1-6dda-49bb-be57-eaaeb79a2a6c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .master('spark://spark-master:7077') \\\n",
    "    .config('spark.jars.packages', 'com.amazonaws:aws-java-sdk-bundle:1.11.469,org.apache.hadoop:hadoop-aws:3.3.1,org.apache.hadoop:hadoop-common:3.3.1') \\\n",
    "    .config('spark.executor.memory', '512m') \\\n",
    "    .config('spark.executor.instances', '2') \\\n",
    "    .config('spark.cores.max', '2') \\\n",
    "    .appName(\"Jup\") \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1922b852-8d56-491d-9233-819ad1f5e564",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3accessKeyAws = \"kGzOWOmt5T8PpVPSffdy\"\n",
    "s3secretKeyAws = \"k8QQDCRknm6C7ypigO9QySO1YQyRgkLHUwRLxIG1\"\n",
    "s3endPointLoc = \"http://minio:9000\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9d07589-067d-4f26-93cf-720e4409f951",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc = spark.sparkContext\n",
    "\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\", s3accessKeyAws)\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\", s3secretKeyAws)\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.endpoint\", s3endPointLoc)\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.path.style.access\", \"true\")\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.connection.ssl.enabled\", \"false\")\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.impl\", \"org.apache.hadoop.fs.s3a.S3AFileSystem\")\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.connection.ssl.enabled\", \"false\")\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.connection.establish.timeout\", \"15000\")\n",
    "sc._jsc.hadoopConfiguration().set(\"fs.s3a.aws.credentials.provider\", \"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8860817f-f65d-4e5b-944e-497408c8d69e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = spark.read.csv('s3a://spark-raw/test.csv', header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cfdda2f-af9e-437c-9290-d3c2583f4064",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ace7862-d27a-442a-a371-24c269d6713d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3013f382-555d-4311-95f2-4c8adf15824e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
